#!/usr/bin/perl

# $Id: ftp_crawler.pl 116 2010-06-26 14:40:54Z dolbanavt $
# $Date: 2010-06-26 17:41:00 +0300 (Суб, 26 Июн 2010) $
# $HeadURL: https://robosearch.googlecode.com/svn/branches/reborn/ftp_crawler.pl $
# $LastChangedRevision$

use lib::abs qw( base_lib lib );
use JIP::ToolSet;
use Carp qw( croak );
use Const::Fast qw( const );
use English qw( -no_match_vars );
use Pod::Usage qw( pod2usage );
use Getopt::Long 2.24 qw( :config no_auto_abbrev no_ignore_case );
use File::Spec::Functions qw( catfile );
use JIP::Utils qw( check_named_params );
use FindBin qw( $RealBin );
use Log::Console;
use Net::FTP;
use FTPWrapper;
use FTPListing::Unix;
use FTPListing::DOS;
use IO::YAML;

our $VERSION = '1.0';

# more meaningfull names for pod2usage's -verbose parameter
const my $SHOW_USAGE_ONLY         => 0;
const my $SHOW_BRIEF_HELP_MESSAGE => 1;
const my $SHOW_FULL_MANUAL        => 2;

# more meaningfull names for pod2usage's -exitval parameter
const my $RETURN_SUCCESS     => 0;
const my $RETURN_FATAL_ERROR => 1;
const my $RETURN_BAD_OPTIONS => 2;

const my $AUTO_TERMINATED_ENABLED => 1; # bool

const my $UNIX_LISTING => qr{
    ^[a-z-]{10}           # drwxr-xr-x
    \s+\d*                # 1
    (?:\s+\w+){2}         # ftp ftp
    \s+\d+                # 0
    \s+\w+\s+\d+\s+[\d:]+ # Feb 15 17:34
    \s+.+$                # linux_format
}xms;

const my $DOS_LISTING => qr{
    ^\d\d-\d\d-\d\d\s+\d\d:\d\d\wM # date and time
    \s+                            # some space
    (?:<DIR>|\d+)                  # dir or size
    \s+                            # some more space
    .+$                            # file name
}xms;

#-- ui -------------------------------------------------------------------------
my $ftp_user    = 'anonymous';        # default settings
my $ftp_pass    = 'upyachka_fighter'; #
my $ftp_timeout = 15;                 #
my $ftp_root    = '/';                #

my $options_okay = GetOptions(
    'help'          => \my $is_help_requested,
    'host=s'        => \my $ftp_host,
    'output_file=s' => \my $path_to_output_file,
    'user=s'        => \$ftp_user,
    'pass=s'        => \$ftp_pass,
    'timeout=i'     => \$ftp_timeout,
    'root_dir=s'    => \$ftp_root,
);

# show appropriate help message
pod2usage( -exitval => $RETURN_BAD_OPTIONS, -verbose => $SHOW_USAGE_ONLY )
    unless $options_okay;

pod2usage( -exitval => $RETURN_SUCCESS, -verbose => $SHOW_FULL_MANUAL )
    if $is_help_requested;

# check required options
pod2usage(
    -exitval => $RETURN_BAD_OPTIONS,
    -message => 'required parameter "host" not defined',
    -verbose => $SHOW_USAGE_ONLY,
) unless $ftp_host;

pod2usage(
    -exitval => $RETURN_BAD_OPTIONS,
    -message => 'required parameter "output_file" not defined',
    -verbose => $SHOW_USAGE_ONLY,
) unless $path_to_output_file;

#-- init log -------------------------------------------------------------------
our $log = Log::Console->new( log_level => 'all' );

#-- signal handlers ------------------------------------------------------------
require( catfile( $RealBin, 'signal_handlers.pl' ) );

#-- FTP-wrapper ----------------------------------------------------------------
$log->Note('init FTP-connection');

my $ftp_connection = init_ftp_connection(
    host    => $ftp_host,
    user    => $ftp_user,
    pass    => $ftp_pass,
    timeout => $ftp_timeout,
);

$log->Note('init listing decorator');

my $listing_decorator = init_listing_decorator_for( $ftp_connection );

$log->Note( 'decorator: '. ref $listing_decorator );

my $ftp_wrapper = FTPWrapper->new(
    connection => $ftp_connection,
    decorator  => $listing_decorator,
);

#-- output object --------------------------------------------------------------
$log->Note('open yaml stream for writing');

my $yaml_writer = IO::YAML->new(
    'path'            => $path_to_output_file,
    'mode'            => 'w',
    'auto_terminated' => $AUTO_TERMINATED_ENABLED,
);

#-- main -----------------------------------------------------------------------
$log->Note('-- seed generator -----------------------------------------------');

$ftp_root = ftp_root_sanity( $ftp_root );

$log->Note( 'root directory: '. $ftp_root );

my @seeds;

foreach ( $ftp_wrapper->get_listing_for( $ftp_root ) ) {
    if ( $ARG->{'type'} eq 'd' ) {
        my $seed = $ftp_root . $ARG->{'name'};

        $log->Note( 'seed: '. $seed ) and push @seeds, $seed;
    }
    elsif ( $ARG->{'type'} eq 'f' ) {
        $yaml_writer->print( {
            type => 'file',
            name => $ARG->{'name'},
            path => $ftp_root,
            size => $ARG->{'size'},
        } ) or croak 'cannot write yaml stream: '. $OS_ERROR;
    }
}

$log->Note('-- crawl loop ---------------------------------------------------');

for my $each_seed ( @seeds ) {
    $log->Note( 'indexing: '. $each_seed ) and indexing_for( $each_seed );
}

$yaml_writer->close() or croak 'cannot close yaml stream: '. $OS_ERROR;

$log->Note('-- exiting ------------------------------------------------------');

#-- subroutines ----------------------------------------------------------------
sub indexing_for {
    my $current_dir = shift;

    $yaml_writer->print( {
        type => 'dir',
        path => $current_dir,
    } ) or croak 'cannot write yaml stream: '. $OS_ERROR;

    foreach ( $ftp_wrapper->get_listing_for( $current_dir ) ) {
        if ( $ARG->{'type'} eq 'f' ) {
            $yaml_writer->print( {
                type => 'file',
                name => $ARG->{'name'},
                path => $current_dir,
                size => $ARG->{'size'},
            } ) or croak 'cannot write yaml stream: '. $OS_ERROR;
        }
        elsif ( $ARG->{'type'} eq 'd' ) {
            indexing_for( $current_dir .'/'. $ARG->{'name'} );
        }
    }
}

sub init_ftp_connection {
    my %param = @ARG;

    check_named_params(
        rules_ref  => {
            host    => { required => 1, defined  => 1 },
            user    => { required => 1, defined  => 1 },
            pass    => { required => 1, defined  => 1 },
            timeout => { required => 1, defined  => 1 },
        },
        params_ref => \%param,
    );

    my $ftp = Net::FTP->new( $param{'host'}, Timeout => $param{'timeout'} )
        or croak 'cannot connect to '. $param{'host'} .': '. $EVAL_ERROR;

    $ftp->login( $param{'user'}, $param{'pass'} )
        or croak 'authentification error: '. $ftp->message;

    return $ftp;
}

sub init_listing_decorator_for {
    my $ftp = shift;

    my @root_dir_listing = $ftp->dir(q{})
        or croak 'cannot get a directory listing: '. $ftp->message;

    croak 'empty directory listing' unless $root_dir_listing[0];

    given ( $root_dir_listing[0] ) {
        when ( $UNIX_LISTING ) { return FTPListing::Unix->new(); }
        when ( $DOS_LISTING  ) { return FTPListing::DOS->new(); }

        default { croak 'unknown system'; }
    }
}

sub ftp_root_sanity {
    my $path = shift;

    $path =~ s!(\/){2,}!\/!xmsg;

    $path =~ m{\/$}xms ? return $path : return $path .'/';
}

#-- end ------------------------------------------------------------------------

__END__

=head1 NAME

ftp_crawler.pl - простой FTP-кроулер

=head1 SYNOPSIS

ftp_crawler.pl [options]

 Options:
   --help            show full help message
   --host            IP-адрес FTP-сервера
   --output_file     путь к файлу с результатами индексирования
   --user            имя пользователя ('anonymous' по умолчанию)
   --pass            пароль пользователя ('upyachka_fighter' по умолчанию)
   --timeout         таймаут подключения к серверу
   --root_dir        директория сервера с которой стартует индексирование

=head1 USAGE

/path/to/ftp_crawler.pl --help

# индексирование локалхоста
/path/to/ftp_crawler.pl --host=127.0.0.1 --output_file=./file.yaml

# индексирование локахоста
# с таймаутом 5 секунд
# индексировать все что в директории /video
/path/to/ftp_crawler.pl --host=127.0.0.1 \
                        --output_file=./file.yaml \
                        --timeout=5 \
                        --root_dir=/video

=head1 DESCRIPTION

Программа для индексирования удаленных FTP-серверов. Рекурсивно обходит
каталоги сервера и сохраняет инфомацию об их содержимом в файл CSV.

=head1 DEPENDENCIES

lib::abs (from CPAN)

Const::Fast (from CPAN)

IO::YAML (from CPAN)

JIP::ToolSet (from base_lib)

JIP::Utils (from base_lib)

Log::Console (from CPAN)

=head1 EXIT STATUS

0 - программа выполнена успешно

1 - критическая ошибка программы

2 - неверные параметры командной строки

=head1 AUTHOR

Vladimir Zhavoronkov, <dolbanavt (at) Google Mail>

http://dolbanavt.livejournal.com/

=head1 LICENSE AND COPYRIGHT

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

=cut
